package com.cmy;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.util.List;

public class IqiyiLiveMobile implements PageProcessor {
    // 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
    private Site site = Site.me()
            .setRetryTimes(3)
            .setSleepTime(100)
            .addHeader("referer", "https://m-live.iqiyi.com/")
//            .addHeader("referer", "https://m-live.iqiyi.com/cate/game")
            .addHeader("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
            .addHeader("cache-control", "no-cache")
            .setUserAgent("Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Mobile Safari/537.36");

    @Override
    // process是定制爬虫逻辑的核心接口，在这里编写抽取逻辑
    public void process(Page page) {
        Html pageHtml = page.getHtml();
        Selectable pageUrl = page.getUrl();
        if (pageUrl.regex("(https://m-live.iqiyi\\.com/cate/game)").all().size() > 0) {
            // 部分二：定义如何抽取页面信息，并保存下来
            List<String> stringList = pageHtml.xpath("/html/body/div/div[2]/ul//li/a/@href").all();
//            for (int i = 0; i < stringList.size(); i++) {
            for (int i = 0; i < 2; i++) {
                // 部分三：从页面发现后续的url地址来抓取
                String target = pageHtml.links() + stringList.get(i);
                page.addTargetRequest(target);
            }

        } else if (pageUrl.regex("(https://m-live.iqiyi\\.com//w/[\\w\\-]+)").all().size() > 0) {

            System.out.println("liveUrl.getHtml : "+page.getHtml());

            // //*[@id="player"]/video
            page.putField("liveUrl", page.getHtml().xpath("//*[@id=\"player\"]/video/@src").toString());
            System.out.println("liveUrl : " + page.getHtml().xpath("//*[@id=\"player\"]/video/@src").toString());
            /*if (page.getResultItems().get("liveUrl") == null) {
                //skip this page
                page.setSkip(true);
            }*/
        }
        // 部分三：从页面发现后续的url地址来抓取
//        page.addTargetRequests(page.getHtml().links().regex("(https://github\\.com/[\\w\\-]+/[\\w\\-]+)").all());
//        System.out.println("page.getResultItems():" + page.getResultItems());
    }

    @Override
    public Site getSite() {
        System.out.println("site : " + site);
        return site;
    }

    public static void main(String[] args) {
        Spider.create(new IqiyiLiveMobile())
                //从"https://github.com/code4craft"开始抓
                .addUrl("https://m-live.iqiyi.com/cate/game")
//                .addPipeline(new ConsolePipeline())
//                .addPipeline(new FilePipeline())
                //开启5个线程抓取
                .thread(3)
                //启动爬虫
                .run();
    }
}
